#!/usr/bin/python3
# ******************************************************************************
# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved.
# licensed under the Mulan PSL v2.
# You can use this software according to the terms and conditions of the Mulan PSL v2.
# You may obtain a copy of Mulan PSL v2 at:
#     http://license.coscl.org.cn/MulanPSL2
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
# PURPOSE.
# See the Mulan PSL v2 for more details.
# ******************************************************************************/
import csv
import json
import os.path
import re
import subprocess
import sys
from collections import defaultdict

from lxml import etree
from requests.exceptions import HTTPError

from python_learn.obs_api.package_relate import PackageRelate
from python_learn.obs_api.project_relate import ProjectRelate


class ParsePackages:
    def __init__(self):
        self.project_client = ProjectRelate()
        self.package_client = PackageRelate()
        # self.project = "openEuler:22.03:LTS"
        # self.project = "home:Lhw-KunPeng:branches:openEuler:22.03:LTS"
        self.project = "openEuler:Factory"
        # self.project = "home:Lhw-KunPeng:branches:openEuler:Mainline"
        self._build_zero = 0
        self._build_less_ten = 0
        self._build_ten_thirty = 0
        self._build_thirty_fifty = 0
        self._build_more_fifty = 0
        self._install_zero = 0
        self._install_less_ten = 0
        self._install_ten_thirty = 0
        self._install_thirty_fifty = 0
        self._install_more_fifty = 0
        self._build_level_dict = {
            "_build_zero": 0,
            "_build_less_ten": 10,
            "_build_ten_thirty": 30,
            "_build_thirty_fifty": 50,
            "_build_more_fifty": sys.maxsize
        }
        self._install_level_dict = {
            "_install_zero": 0,
            "_install_less_ten": 10,
            "_install_ten_thirty": 30,
            "_install_thirty_fifty": 50,
            "_install_more_fifty": sys.maxsize
        }

    @staticmethod
    def file_path(file_name):
        return os.path.join('data', file_name)

    def get_all_packages_reuqireby(self):
        lg_15 = 0
        lg_10_lt_15 = 0
        lt_10 = 0
        lg_10_list = []
        all_packages = self._get_all_pkgs_of_project()
        for package in all_packages:
            src_and_bin_rpms = self._get_src_and_bin_rpm(package)
            required_rpms = []
            self._get_requiredby_rpm(package, required_rpms, src_and_bin_rpms)
            required_dict = {package: required_rpms}

            if len(required_rpms) >= 15:
                lg_15 += 1
                print(required_dict)
                lg_10_list.append(required_dict)
                continue
            if len(required_rpms) >= 10:
                lg_10_lt_15 += 1
                print(required_dict)
                lg_10_list.append(required_dict)
                continue
            lt_10 += 1

        print(f'>15: {lg_15}; 10-15: {lg_10_lt_15}; <10: {lt_10}')

        with open(self.file_path(self.file_path('rpms.json')), 'w') as file:
            json.dump(lg_10_list, file)

    def get_all_packages_dnf_reuqireby(self):
        """
        通过dnf方式分析所有软件包的被依赖
        :return:
        """
        all_packages = self._get_all_pkgs_of_project()
        requiredby_all_rpms = dict()
        try:
            for package in all_packages:
                # 中间执行时使用
                # if ord(package[0]) - ord('a') < 12:
                #     continue
                requiredby_rpms_set = set()
                try:
                    src_and_bin_rpms = self._get_src_and_bin_rpm(package)
                except HTTPError:
                    continue
                else:
                    for rpm in src_and_bin_rpms:
                        rpm = str(rpm).split('.oe1')[0] + '.oe1'
                        cmd = ["dnf", "repoquery", "--whatdepends", rpm]
                        cmd_result = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=False)
                        cmd_result_text = cmd_result.decode('utf-8')
                        requiredby_rpms = cmd_result_text.split('\n')[1:]
                        for req_rpm in requiredby_rpms:
                            if req_rpm and req_rpm.find('.src') > 0:
                                requiredby_rpms_set.add(re.split('-\d+:', req_rpm)[0])
                requiredby_all_rpms[package] = list(requiredby_rpms_set)
                print(f'{package} requiredby rpms: {requiredby_rpms_set}')
        except Exception:
            print('Error Continue')

        with open(self.file_path(f'dnf_requiredby_rpms-{self.project}.json'), 'w') as f:
            json.dump(requiredby_all_rpms, f)
        #         provide_components = list()
        #         self._get_provide_components(package=package, provide_components=provide_components,
        #                                      src_and_bin_rpms=src_and_bin_rpms)
        #         for component in provide_components:
        #             cmd = ["dnf", "repoquery", "--whatdepends", component]
        #             cmd_result = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=False)
        #             cmd_result_text = cmd_result.decode('utf-8')
        #             all_count = len(cmd_result_text.split('\n')) - 2
        #             build_count = cmd_result_text.count(".src")
        #             install_count = all_count - build_count
        #             requiredby_count_of_provides[f'{package}:{component}:build'] = build_count
        #             requiredby_count_of_provides[f'{package}:{component}:install'] = install_count
        #             self._count_statistic(build_count, require_type="build")
        #             self._count_statistic(install_count, require_type="install")
        #     except HTTPError:
        #         continue
        #
        #     print({"_build_less_ten": self._build_less_ten,
        #            "_build_ten_thirty": self._build_ten_thirty,
        #            "_build_thirty_fifty": self._build_thirty_fifty,
        #            "_build_more_fifty": self._build_more_fifty,
        #            "_install_less_ten": self._install_less_ten,
        #            "_install_ten_thirty": self._install_ten_thirty,
        #            "_install_thirty_fifty": self._install_thirty_fifty,
        #            "_install_more_fifty": self._install_more_fifty, })
        #
        # all_result = [requiredby_count_of_provides, {"_build_less_ten": self._build_less_ten,
        #                                              "_build_ten_thirty": self._build_ten_thirty,
        #                                              "_build_thirty_fifty": self._build_thirty_fifty,
        #                                              "_build_more_fifty": self._build_more_fifty,
        #                                              "_install_less_ten": self._install_less_ten,
        #                                               "_install_ten_thirty": self._install_ten_thirty,
        #                                              "_install_thirty_fifty": self._install_thirty_fifty,
        #                                              "_install_more_fifty": self._install_more_fifty, }]
        # with open(self.file_path('dnf_requiredby_count.json'), 'w') as f:
        #     json.dump(all_result, f)

    def get_provides_requiredby_rpms(self):
        """
        获取软件包各个组件的被依赖数据
        :return:
        """
        with open(self.file_path('rpms_count.json'), 'r') as file:
            rpm_count_data = json.load(file)

        more_50_rpms_info = rpm_count_data.get('50_100', [])
        more_50_rpms_info.extend(rpm_count_data.get('lg_100', []))
        more_50_rpms_name = []
        for rpm_info in more_50_rpms_info:
            more_50_rpms_name.extend(list(rpm_info.keys()))

        # more_50_rpms_name = ['python-flask']
        result = []
        for package in more_50_rpms_name:
            src_and_bin_rpms = self._get_src_and_bin_rpm(package=package)
            provides_and_requiredby_rpms = dict()
            for rpm in src_and_bin_rpms:
                requiredby_xml = self.package_client.get_package_provides_and_requiredby(rpm=rpm, package=package,
                                                                                         target_project=self.project)
                provides_and_requiredby_rpms.update(self._parse_provides_requiredby_xml(requiredby_xml)[1])
            package_provides_and_requiredby_rpms = {package: provides_and_requiredby_rpms}
            print(package_provides_and_requiredby_rpms)
            result.append(package_provides_and_requiredby_rpms)

        with open(self.file_path('package_provides_required.json'), 'w') as file:
            json.dump(result, file)

    def count_more_required(self, file_name, result_file_name):
        """
        按照被依赖包数量归类
        :return:
        """
        with open(self.file_path(file_name), 'r') as f:
            json_data = json.load(f)

        result = defaultdict(list)

        self._level_count(json_data, result)

        with open(self.file_path(result_file_name), 'w') as f:
            json.dump(result, f)

    def count_more_provides_requiredby(self):
        with open(self.file_path('provides_required.json'), 'r') as f:
            json_data = json.load(f)
        result = defaultdict(list)
        self._level_count(json_data, result)

        with open(self.file_path('provides_required_count.json'), 'w') as f:
            json.dump(result, f)

    def convert_json_to_csv(self, json_file_name, csv_file_name):
        """
        转化json为csv
        :param json_file_name:
        :param csv_file_name:
        :return:
        """
        try:
            with open(self.file_path(json_file_name), 'r') as json_file:
                json_data = json.load(json_file)

            with open(self.file_path(csv_file_name), 'w') as csv_file:
                csv_writer = csv.writer(csv_file)
                for level, count_detail in json_data.items():
                    for info in count_detail:
                        for rpm_name, count in info.items():
                            csv_writer.writerow([rpm_name, count])
        except FileNotFoundError:
            print('file not found')

    def _get_requiredby_rpm(self, package, required_rpms, src_and_bin_rpms):
        for rpm in src_and_bin_rpms:
            requiredby_xml = self.package_client.get_package_provides_and_requiredby(rpm=rpm, package=package,
                                                                                     target_project=self.project)
            requiredby_rpms = self._parse_requiredby_xml(requiredby_xml)
            required_rpms.extend(set(requiredby_rpms))

    def _get_provide_components(self, package, provide_components, src_and_bin_rpms):
        for rpm in src_and_bin_rpms:
            requiredby_xml = self.package_client.get_package_provides_and_requiredby(rpm=rpm, package=package,
                                                                                     target_project=self.project)
            provide_components.extend(self._parse_provides_requiredby_xml(requiredby_xml)[0])

    def _get_src_and_bin_rpm(self, package):
        src_bin_xml = self.package_client.get_package_src_and_bin(package=package, target_project=self.project)
        src_and_bin_rpms = self._parse_src_bin_xml(src_bin_xml)
        return src_and_bin_rpms

    def _get_all_pkgs_of_project(self):
        packages_xml = self.project_client.get_project_packages(project=self.project)
        all_packages = self._parse_packages_xml(old_xml=packages_xml)
        return all_packages

    def _parse_packages_xml(self, old_xml):
        elements_tree = etree.HTML(old_xml)
        entries = elements_tree.xpath("//entry//@name")
        return entries

    def _parse_src_bin_xml(self, src_bin_xml):
        elements_tree = etree.HTML(src_bin_xml)
        binaries = elements_tree.xpath("//binary//@filename")
        return [binary for binary in binaries if binary != "_buildenv" and binary != "_statistics"]

    def _parse_requiredby_xml(self, requiredby_xml):
        elements_tree = etree.HTML(requiredby_xml)
        rpms = elements_tree.xpath("//provides_ext/requiredby/@name")
        return rpms

    def _parse_provides_requiredby_xml(self, requiredby_xml):
        elements_tree = etree.HTML(requiredby_xml)
        provides_info = elements_tree.xpath("//provides_ext")
        provide_components = list()
        provides_requiredby = dict()

        for provides in provides_info:
            dep = provides.xpath("@dep")[0]
            provide_components.append(dep)
            requiredby_rpms = provides.xpath("//requiredby/@name")
            requiredby_rpms.append(len(requiredby_rpms))
            provides_requiredby[dep] = requiredby_rpms

        return provide_components, provides_requiredby

    def _level_count(self, json_data, static_result, require_type="build"):
        if isinstance(json_data, list):
            for data in json_data:
                self._level_count_detail(data, static_result, require_type)
        elif isinstance(json_data, dict):
            self._level_count_detail(json_data, static_result, require_type)

    def _level_count_detail(self, data, static_result, require_type="build"):
        iter_count_type_dict = self._build_level_dict if require_type == "build" else self._install_level_dict
        for key, value in data.items():
            value_set = set(value)
            required_count = len(value_set)
            count_dict = {key: required_count}
            for level_name, level_count in iter_count_type_dict.items():
                if required_count <= level_count:
                    static_result[level_name].append(count_dict)
                    break

    def _count_statistic(self, requiredby_count, require_type):
        if requiredby_count == 0:
            return
        if 0 < requiredby_count < 10:
            old_count = getattr(self, f'_{require_type}_less_ten')
            setattr(self, f'_{require_type}_less_ten', old_count + 1)
            return
        if 10 <= requiredby_count <= 30:
            old_count = getattr(self, f'_{require_type}_ten_thirty')
            setattr(self, f'_{require_type}_ten_thirty', old_count + 1)
            return
        if 30 < requiredby_count <= 50:
            old_count = getattr(self, f'_{require_type}_thirty_fifty')
            setattr(self, f'_{require_type}_thirty_fifty', old_count + 1)
            return
        if requiredby_count > 50:
            old_count = getattr(self, f'_{require_type}_more_fifty')
            setattr(self, f'_{require_type}_more_fifty', old_count + 1)


if __name__ == '__main__':
    parsePackage = ParsePackages()
    ## 获取rpm级别的被依赖关系
    # parsePackage.get_all_packages_dnf_reuqireby()
    # parsePackage.count_more_required(file_name='dnf_requiredby_rpms-openEuler:Factory.json',
    #                                  result_file_name='dnf_requiredby_count-openEuler:Factory.json')
    parsePackage.convert_json_to_csv(json_file_name='dnf_requiredby_count-openEuler:Factory.json',
                                     csv_file_name='dnf_requiredby_count-openEuler:Factory.csv')
    # parsePackage.get_all_packages_reuqireby()
    # parsePackage.count_more_required()
    # parsePackage.get_provides_requiredby_rpms()
    # parsePackage.count_more_provides_requiredby()
    # result_list = list()
    # parsePackage.get_requiredby_rpm(package="python-mock", required_rpms=result_list,
    #                                 src_and_bin_rpms=["python3-mock-4.0.3-1.oe1.noarch.rpm"])
    # print(result_list)
